Codec Evaluation for Robotics

Jacky Baltes
National Taiwan Normal University
Taipei, Taiwan
jacky.baltes@ntnu.edu.tw

09 April 2020

Test Video - Robinion Walking - 90 Deg.

import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import cm

cap = cv2.VideoCapture( '/content/BuildDir/reveal.js/assets/videos/robinion1.mp4' )

fig = plt.figure( figsize=(10,10) )
ax = fig.add_subplot(1,1,1)

frames = []

NFRAMES = 15*30

count = 0
while(True):
    # Capture frame-by-frame
    ret, frame = cap.read( )

    if not ret or frame is None or count >= NFRAMES:
      break
    # print( "Frame", count )
    # Our operations on the frame come here
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Display the resulting frame
    img = ax.imshow( gray, cmap='gray' )
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    count = count + 1
    frames.append( [img] )

ani = animation.ArtistAnimation(fig, frames, interval=50, blit=True,
                                repeat_delay=1000)

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()

aniWalk = addJBAnimation( "aniWalk", 0, 0, ani );
print()

Input Video

# Subsample routine of numpy arrays
def subSample( img, sub = 4 ):
  return img[::sub, ::sub]
class Codec:
  def __init__(self, quality):
    self.quality = quality

  def encode(self, img ):
    pass

  def decode( self, img ):
    pass

  def test( self, img ):
    f = self.encode( img )
    fDec = self.decode( f )

    fheight = min( img.shape[0], fDec.shape[0] )
    fwidth = min( img.shape[1], fDec.shape[1] )
    
    errorFrame = np.sqrt( np.sum( ( img[0:fheight,0:fwidth] - fDec[0:fheight,0:fwidth] ) ** 2, axis=-1 ) )

    return ( f, fDec, errorFrame )
class CodecNearestNeighbor(Codec):
  def encode(self, img ):
    return img[::self.quality,::self.quality]    

  def decode(self, img ):
    height, width, depth = img.shape
    out = np.zeros( ( height * self.quality, width * self.quality, depth ), dtype = img.dtype )
    for y in range( height ):
      for x in range( width ):
        out[y*self.quality:y*self.quality+self.quality, x*self.quality:x*self.quality+self.quality ] = img[y,x]
    return out
frame = savedImgs[0]

sampling = [1, 2, 4, 8, 16 ]

fig = plt.figure( figsize=(12,30) )
axes=fig.subplots(5,1)

for i,s in enumerate( sampling ):
  codec = CodecNearestNeighbor( s )
  f, fDec, errorFrame = codec.test( frame )
  axes[i].imshow( fDec )
  
  error = np.sum( errorFrame )
  axes[i].imshow( errorFrame, alpha=0.7, cmap=cm.Reds )

  axes[i].text( 200, 60, f"Ratio:{(f.shape[0] * f.shape[1])/(frame.shape[0]*frame.shape[1]):5.4f}\nError: {error}\nNormalized: {error/(frame.shape[0]*frame.shape[1]):5.2f}", color="#80ff2020", fontsize=16 )
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import cm

#from google.colab.patches import cv2_imshow

cap = cv2.VideoCapture( '/content/BuildDir/reveal.js/assets/videos/robinion1.mp4' )

fig = plt.figure( figsize=(12,10) )
axes = fig.subplots(2,2)
ax1 = axes[0][0]
ax2 = axes[1][0]
ax3 = axes[0][1]
ax4 = axes[1][1]

frames = []

NFRAMES = 15 * 30

count = 0

savedImgs = []
errorLists = [ [], [], [] ]

while(True):
    # Capture frame-by-frame
    artists = []
    ret, frameIn = cap.read( )

    if not ret or frameIn is None or count >= NFRAMES:
      break

    #print("Frame", count )
    frame = subSample( frameIn, 4 )

    codec1 = CodecNearestNeighbor( 2 )
    fEnc, fDec, fError = codec1.test( frame )
    error = np.sum(fError) / ( fDec.shape[0] * fDec.shape[1] )
    errorLists[0].append( error )

    # Display the resulting frame
    img = ax1.imshow( frame )
    img2 = ax1.imshow( fError, alpha=0.5, cmap=cm.Reds)
    iText = ax1.text( 160, 60, f"Ratio:{(fEnc.shape[0] * fEnc.shape[1])/(frame.shape[0]*frame.shape[1]):5.4f}\nRMS Error: {error:5.2f}", color="#80ff2020", fontsize=16 )

    plot1, = ax2.plot( errorLists[0], 'b-', linewidth=2, label="Factor: 2" )

    artists.extend( [img, img2, iText, plot1 ] )    

    codec2 = CodecNearestNeighbor( 8 )
    fEnc, fDec, fError = codec2.test( frame )
    error = np.sum(fError) / ( fDec.shape[0] * fDec.shape[1] )
    errorLists[1].append( error )

    # Display the resulting frame
    img = ax3.imshow( frame )
    img2 = ax3.imshow( fError, alpha=0.5, cmap=cm.Reds)
    iText = ax3.text( 160, 60, f"Ratio:{(fEnc.shape[0] * fEnc.shape[1])/(frame.shape[0]*frame.shape[1]):5.4f}\nRMS Error: {error:5.2f}", color="#80ff2020", fontsize=16 )

    plot1, = ax2.plot( errorLists[1], 'g-', linewidth=2, label="Factor: 8" )
    artists.extend( [img, img2, iText, plot1 ] )

    codec3 = CodecNearestNeighbor( 16 )
    fEnc, fDec, fError = codec3.test( frame )
    error = np.sum(fError) / ( fDec.shape[0] * fDec.shape[1] )
    errorLists[2].append( error )

    # Display the resulting frame
    img = ax4.imshow( frame )
    img2 = ax4.imshow( fError, alpha=0.5, cmap=cm.Reds)
    iText = ax4.text( 160, 60, f"Ratio:{(fEnc.shape[0] * fEnc.shape[1])/(frame.shape[0]*frame.shape[1]):5.4f}\nRMS Error: {error:5.2f}", color="#80ff2020", fontsize=16 )

    plot1, = ax2.plot( errorLists[2], 'r-', linewidth=2, label="Factor: 16" )
    artists.extend( [img, img2, iText, plot1 ] )

    frames.append( artists )

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    count = count + 1

# l = ax2.legend()
    
ani = animation.ArtistAnimation(fig, frames, interval=50, blit=True,
                                repeat_delay=1000)

# ani.save('dynamic_images.mp4')

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()

#createBase64VideoFromAnimation
#HTML(ani.to_html5_video())
ani1 = addJBAnimation( "ani1", 0, 0, ani )

RMS Comparison

class CodecAverage(Codec):
  def encode(self, img ):
    out = np.zeros( ( img.shape[0] // self.quality, img.shape[1] // self.quality, img.shape[2] ), dtype = img.dtype )
    height, width, depth = out.shape
    for y in range( height ):
      for x in range( width ):
        out[y,x] = np.average( img[y*self.quality:y*self.quality+self.quality, 
                                   x*self.quality:x*self.quality+self.quality ], axis=(0,1) )
    return out

  def decode(self, img ):
    height, width, depth = img.shape
    out = np.zeros( ( height * self.quality, width * self.quality, depth ), dtype = img.dtype )
    for y in range( height ):
      for x in range( width ):
        out[y*self.quality:y*self.quality+self.quality, x*self.quality:x*self.quality+self.quality ] = img[y,x]
    return out

RMS Comparison

.